package com.example.webmagic;


import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.*;


import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;

import static org.apache.logging.log4j.ThreadContext.isEmpty;

/**
 * 抓取图片
 *
 * @author yzh
 * @date 2022/12/06
 */
public class Crawl_Image implements PageProcessor {
    static Logger logger = LoggerFactory.getLogger(Crawl_Image.class);

 /*  Site.me()
        .setCharset("UTF-8")//编码
        .setSleepTime(1)//抓取间隔时间
        .setTimeOut(1000*10)//超时时间
        .setRetrySleepTime(3000)//重试时间
        .setRetryTimes(3);//重试次数*/
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

    @Override
    public void process(Page page) {
        /*this.getImage(page);*/
        String s = page.getHtml().xpath("//li[@class='nextpage']/a/@href").get();
        /*System.out.println(page.getHtml());*/
        String url = "";
        if (s != null) {
            url = page.getHtml().xpath("//li[@class='nextpage']/a/@href").get();
            this.getImage(page);
            page.addTargetRequest(url);
        }
        /* this.getImage(page);*/
    }

    @Override
    public Site getSite() {
        return site;
    }


    /**
     * 爬图片的url和title
     */
    public void getImage(Page page) {
        List<String> imageUrl = page.getHtml().xpath("//div[@class='list list-phone']/ul/li/a/img/@src").all();
        List<String> title = page.getHtml().xpath("//div[@class='list list-phone']/ul/li/a/img/@alt").all();

        page.putField("urlString", imageUrl);
        page.putField("fileName", title);
    }

    public static void main(String[] args) throws IOException {

        //爬取动态页面需要浏览器进行操作
        System.setProperty("selenuim_config", "E:\\项目\\Mydemo\\webmagic\\src\\main\\resources\\config.ini");


        SeleniumDownloader seleniumDownloader = new SeleniumDownloader("C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
        seleniumDownloader.setSleepTime(3000);//设置延迟


        Spider.create(new Crawl_Image()).addUrl("http://www.netbian.com/shouji/meinv/index_2.html")
                .setDownloader(seleniumDownloader)
                .addPipeline(new ConsolePipeline())//打印至控制台
                .addPipeline(new Image_Pipeline())
                .thread(50).run();
        /* download("https://tiebapic.baidu.com/forum/w%3D580/sign=47f50bd8dcef76093c0b99971edda301/cfc179310a55b319a0c58d3054a98226cffc17bc.jpg?tbpicau=2022-12-08-05_1957b38b2220f555a70ed988085cc39c", "d:\\web\\", "sample.png");
         */
    }


    /**
     * 自定义保存下载类
     */
    public static class Image_Pipeline implements Pipeline {

        @Override
        public void process(ResultItems resultItems, Task task) {
            List<String> urlString = resultItems.get("urlString");
            List<String> fileName = resultItems.get("fileName");

           /* String urlString = resultItems.get("urlString");
            String fileName=resultItems.get("fileName");*/

            // 构造URL
            try {
                for (int i = 0; i < urlString.size(); i++) {

                    URL url = new URL(urlString.get(i));
                    // 打开连接
                    URLConnection con = url.openConnection();
                    //设置请求超时为20s
                    con.setConnectTimeout(5 * 1000);
                    //文件路径不存在 则创建
                    File file = new File("D:\\web\\图片\\");
                    if (!file.exists()) {
                        file.mkdir();
                    }

                    InputStream in = con.getInputStream();
                    OutputStream out = new FileOutputStream(file.getPath() + "\\" + fileName.get(i) + ".jpg");
                    //创建缓冲区
                    byte[] buff = new byte[1024];
                    int n;
                    // 开始读取
                    while ((n = in.read(buff)) >= 0) {
                        out.write(buff, 0, n);
                    }
                }
            } catch (Exception e) {
                /* e.printStackTrace();*/
            }
        }
    }


    public static void download(String urlString, String path, String fileName) throws IOException {
        // 构造URL
        URL url = new URL(urlString);
        // 打开连接
        URLConnection con = url.openConnection();
        //设置请求超时为20s
        con.setConnectTimeout(20 * 1000);
        //文件路径不存在 则创建
        File file = new File(path);
        if (!file.exists()) {
            file.mkdir();
        }

        InputStream in = con.getInputStream();
        OutputStream out = new FileOutputStream(file.getPath() + "\\" + fileName);
        //创建缓冲区
        byte[] buff = new byte[1024];
        int n;
        // 开始读取
        while ((n = in.read(buff)) >= 0) {
            out.write(buff, 0, n);
        }
    }
}
